{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "print(datetime.now())\n",
    "#data preprocessing\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "import collections\n",
    "from collections import defaultdict\n",
    "# NN\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import time\n",
    "import math\n",
    "from sklearn.calibration import calibration_curve\n",
    "from sklearn.metrics import roc_curve, precision_recall_curve, f1_score, roc_auc_score, auc, accuracy_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.impute import SimpleImputer\n",
    "import matplotlib.lines as mlines\n",
    "import matplotlib.transforms as mtransforms\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from captum.attr import IntegratedGradients\n",
    "\n",
    "# the full files pathes are here\n",
    "DATA_PATH_stages=\"../data/kdigo_stages_measured.csv\" \n",
    "DATA_PATH_labs = \"../data/labs-kdigo_stages_measured.csv\" \n",
    "DATA_PATH_vitals = \"../data/vitals-kdigo_stages_measured.csv\" \n",
    "DATA_PATH_vents = \"../data/vents-vasopressor-sedatives-kdigo_stages_measured.csv\"\n",
    "DATA_PATH_detail=\"../data/icustay_detail-kdigo_stages_measured.csv\" \n",
    "SEPARATOR=\";\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMPUTE_EACH_ID = False # imputation within each icustay_id with most common value\n",
    "IMPUTE_COLUMN = False # imputation based on whole column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set parameter as constant \n",
    "\n",
    "TESTING = False \n",
    "TEST_SIZE = 0.05\n",
    "\n",
    "SPLIT_SIZE = 0.2 \n",
    "MAX_DAYS = 35\n",
    "\n",
    "#which classifier to use, only run one classifier at one time \n",
    "CLASS1 = True   #AnyAKI\n",
    "#CLASS2 = False    #ModerateSevereAKI\n",
    "#CLASS3 = False    #SevereAKI\n",
    "ALL_STAGES = False # not binary label, each class separately 0,1,2,3\n",
    "    \n",
    "MAX_FEATURE_SET = True\n",
    "#DIAGNOSIS = False\n",
    "\n",
    "FIRST_TURN_POS = True # creating one label per one ICU stay id\n",
    "\n",
    "# resampling  and imputing\n",
    "TIME_SAMPLING = True \n",
    "SAMPLING_INTERVAL = '6H'\n",
    "RESAMPLE_LIMIT = 16 # 4 days*6h interval\n",
    "MOST_COMMON = False #resampling with most common\n",
    "# if MOST_COMMON is not applied,sampling with different strategies per kind of variable, \n",
    "# numeric variables use mean value, categorical variables use max value\n",
    "\n",
    "IMPUTE_METHOD = 'most_frequent' \n",
    "FILL_VALUE = 0 #fill missing value and ragged part of 3d array\n",
    "\n",
    "#Age constraints: adults\n",
    "ADULTS_MIN_AGE = 18\n",
    "ADULTS_MAX_AGE = -1\n",
    "\n",
    "NORMALIZATION = 'min-max' \n",
    "\n",
    "CAPPING_THRESHOLD_UPPER = 0.99\n",
    "CAPPING_THRESHOLD_LOWER = 0.01\n",
    "\n",
    "# How much time the prediction should occur (hours)\n",
    "HOURS_AHEAD = 48\n",
    "\n",
    "NORM_TYPE = 'min_max'\n",
    "\n",
    "RANDOM = 42\n",
    "\n",
    "#set changable info corresponding to each classifier as variables\n",
    "\n",
    "min_set =  [\"icustay_id\", \"charttime\", \"creat\", \"uo_rt_6hr\", \"uo_rt_12hr\", \"uo_rt_24hr\", \"aki_stage\"]\n",
    "\n",
    "max_set = ['icustay_id', 'charttime', 'aki_stage', 'hadm_id','aniongap_avg', 'bicarbonate_avg', \n",
    "           'bun_avg','chloride_avg', 'creat', 'diasbp_mean', 'glucose_avg', 'heartrate_mean',\n",
    "           'hematocrit_avg', 'hemoglobin_avg', 'potassium_avg', 'resprate_mean','sodium_avg', 'spo2_mean', 'sysbp_mean', \n",
    "           'uo_rt_12hr', 'uo_rt_24hr','uo_rt_6hr', 'wbc_avg', 'sedative', 'vasopressor', 'vent', 'age', 'F','M', \n",
    "           'asian', 'black', 'hispanic', 'native', 'other', 'unknown','white', 'ELECTIVE', 'EMERGENCY', 'URGENT']\n",
    "\n",
    "# LSTM\n",
    "batch_size = 5\n",
    "\n",
    "# naming model and plot\n",
    "classifier_name = \"None vs. Any AKI\"    ###change every time #Moderate vs. Severe #None vs. Any #Others vs. Severe\n",
    "plot_name = \"adult_AnyAKI_LR\"    ###change every time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some functions used later\n",
    "\n",
    "def cap_data(df):\n",
    "    print(\"Capping between the {} and {} quantile\".format(CAPPING_THRESHOLD_LOWER, CAPPING_THRESHOLD_UPPER))\n",
    "    cap_mask = df.columns.difference(['icustay_id', 'charttime', 'aki_stage'])\n",
    "    df[cap_mask] = df[cap_mask].clip(df[cap_mask].quantile(CAPPING_THRESHOLD_LOWER),\n",
    "                                     df[cap_mask].quantile(CAPPING_THRESHOLD_UPPER),\n",
    "                                     axis=1)\n",
    "\n",
    "    return df\n",
    " \n",
    "    \n",
    "def normalise_data(df, norm_mask):\n",
    "    print(\"Normalizing in [0,1] with {} normalization\".format(NORMALIZATION))\n",
    "    \n",
    "    df[norm_mask] = (df[norm_mask] - df[norm_mask].min()) / (df[norm_mask].max() - df[norm_mask].min())\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "# impute missing value in resampleing data with most common based on each id\n",
    "def fast_mode(df, key_cols, value_col):\n",
    "    \"\"\" Calculate a column mode, by group, ignoring null values. \n",
    "    \n",
    "    key_cols : list of str - Columns to groupby for calculation of mode.\n",
    "    value_col : str - Column for which to calculate the mode. \n",
    "\n",
    "    Return\n",
    "    pandas.DataFrame\n",
    "        One row for the mode of value_col per key_cols group. If ties, returns the one which is sorted first. \"\"\"\n",
    "    return (df.groupby(key_cols + [value_col]).size() \n",
    "              .to_frame('counts').reset_index() \n",
    "              .sort_values('counts', ascending=False) \n",
    "              .drop_duplicates(subset=key_cols)).drop('counts',axis=1)\n",
    "\n",
    "\n",
    "#get max shape of 3d array\n",
    "def get_dimensions(array, level=0):   \n",
    "    yield level, len(array)\n",
    "    try:\n",
    "        for row in array:\n",
    "            yield from get_dimensions(row, level + 1)\n",
    "    except TypeError: #not an iterable\n",
    "        pass\n",
    "\n",
    "def get_max_shape(array):\n",
    "    dimensions = defaultdict(int)\n",
    "    for level, length in get_dimensions(array):\n",
    "        dimensions[level] = max(dimensions[level], length)\n",
    "    return [value for _, value in sorted(dimensions.items())]\n",
    "\n",
    "#pad the ragged 3d array to rectangular shape based on max size\n",
    "def iterate_nested_array(array, index=()):\n",
    "    try:\n",
    "        for idx, row in enumerate(array):\n",
    "            yield from iterate_nested_array(row, (*index, idx)) \n",
    "    except TypeError: # final level            \n",
    "        yield (*index, slice(len(array))), array # think of the types\n",
    "\n",
    "def pad(array, fill_value):\n",
    "    dimensions = get_max_shape(array)\n",
    "    result = np.full(dimensions, fill_value, dtype = np.float64)  \n",
    "    for index, value in iterate_nested_array(array):\n",
    "        result[index] = value \n",
    "    return result\n",
    "\n",
    "def bin_total(y_true, y_prob, n_bins):\n",
    "    bins = np.linspace(0., 1. + 1e-8, n_bins + 1)\n",
    "\n",
    "    # In sklearn.calibration.calibration_curve,\n",
    "    # the last value in the array is always 0.\n",
    "    binids = np.digitize(y_prob, bins) - 1\n",
    "\n",
    "    return np.bincount(binids, minlength=len(bins))\n",
    "\n",
    "def missing_bin(bin_array):\n",
    "    midpoint = \" \"    \n",
    "    if bin_array[0]==0:\n",
    "        midpoint = \"5%, \"\n",
    "    if bin_array[1]==0:\n",
    "        midpoint = midpoint + \"15%, \"\n",
    "    if bin_array[2]==0:\n",
    "        midpoint = midpoint + \"25%, \"\n",
    "    if bin_array[3]==0:\n",
    "        midpoint = midpoint + \"35%, \" \n",
    "    if bin_array[4]==0:\n",
    "        midpoint = midpoint + \"45%, \"\n",
    "    if bin_array[5]==0:\n",
    "        midpoint = midpoint + \"55%, \"\n",
    "    if bin_array[6]==0:\n",
    "        midpoint = midpoint + \"65%, \"\n",
    "    if bin_array[7]==0:\n",
    "        midpoint = midpoint + \"75%, \"\n",
    "    if bin_array[8]==0:\n",
    "        midpoint = midpoint + \"85%, \"\n",
    "    if bin_array[9]==0:\n",
    "        midpoint = midpoint + \"95%, \"\n",
    "    return \"The missing bins have midpoint values of \"+ str(midpoint)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"read csv files\")\n",
    "#reading csv files\n",
    "X = pd.read_csv(DATA_PATH_stages, sep= SEPARATOR)\n",
    "X.drop([\"aki_stage_creat\", \"aki_stage_uo\"], axis = 1, inplace = True)\n",
    "#remove totally empty rows \n",
    "X = X.dropna(how = 'all', subset = ['creat','uo_rt_6hr','uo_rt_12hr','uo_rt_24hr','aki_stage'])\n",
    "print(\"convert charttime to timestamp\")\n",
    "X['charttime'] = pd.to_datetime(X['charttime'])\n",
    "\n",
    "#merge rows if they have exact timestamp within same icustay_id AL : it substitutes missing values with zero\n",
    "#X = X.groupby(['icustay_id', 'charttime']).sum().reset_index(['icustay_id', 'charttime'])\n",
    "\n",
    "dataset_detail = pd.read_csv(DATA_PATH_detail, sep= SEPARATOR)  #age constraint\n",
    "dataset_detail.drop(['dod', 'admittime','dischtime', 'los_hospital','ethnicity','hospital_expire_flag', 'hospstay_seq',\n",
    "       'first_hosp_stay', 'intime', 'outtime', 'los_icu', 'icustay_seq','first_icu_stay'], axis = 1, inplace = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_labs = pd.read_csv(DATA_PATH_labs, sep= SEPARATOR) # 'bands lactate platelet ptt inr pt\n",
    "dataset_labs.drop(['albumin_min', 'albumin_max','bilirubin_min', 'bilirubin_max','bands_min', 'bands_max',\n",
    "                   'lactate_min', 'lactate_max','platelet_min', 'platelet_max','ptt_min', 'ptt_max', \n",
    "                   'inr_min', 'inr_max', 'pt_min', 'pt_max'], axis = 1, inplace = True)\n",
    "dataset_labs = dataset_labs.dropna(subset=['charttime'])\n",
    "dataset_labs = dataset_labs.dropna(subset=dataset_labs.columns[4:], how='all')\n",
    "dataset_labs['charttime'] = pd.to_datetime(dataset_labs['charttime'])\n",
    "dataset_labs = dataset_labs.sort_values(by=['icustay_id', 'charttime'])\n",
    "\n",
    "if  MAX_FEATURE_SET:\n",
    "    dataset_vitals = pd.read_csv(DATA_PATH_vitals, sep= SEPARATOR)  \n",
    "    dataset_vents = pd.read_csv(DATA_PATH_vents , sep= SEPARATOR)\n",
    "    #dataset_icd = pd.read_csv(DATA_PATH_icd, sep= SEPARATOR)\n",
    "    dataset_vitals.drop([\"heartrate_min\", \"heartrate_max\",\"sysbp_min\", \"sysbp_max\",\"diasbp_min\", \"diasbp_max\",\n",
    "                        'meanbp_min','meanbp_max', 'meanbp_mean','tempc_min', 'tempc_max', 'tempc_mean',\n",
    "                        \"resprate_min\", \"resprate_max\", \"spo2_min\", \"spo2_max\", \"glucose_min\", \"glucose_max\"], axis = 1, inplace = True)    \n",
    "    print(\"convert charttime to timestamp\")\n",
    "    dataset_vitals['charttime'] = pd.to_datetime(dataset_vitals['charttime'])\n",
    "    dataset_vents['charttime'] = pd.to_datetime(dataset_vents['charttime'])\n",
    "    dataset_vitals = dataset_vitals.sort_values(by=['icustay_id', 'charttime'])\n",
    "    dataset_vents = dataset_vents.sort_values(by=['icustay_id', 'charttime'])\n",
    "    # AL drop those where all columns are nan (empty rows)\n",
    "    dataset_vitals = dataset_vitals.dropna(subset=dataset_vitals.columns[4:], how='all')   \n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(datetime.now())\n",
    "# Labs file: instead of min and max their avg\n",
    "counter = 0\n",
    "col1 = 4\n",
    "col2 = 5\n",
    "null_l = [] # no null values in those that are different\n",
    "changed = 0 # 4316 records changed to avg\n",
    "\n",
    "while counter < 11:\n",
    "    row = 0\n",
    "# find where min and max are different and save their row indices \n",
    "    while row < len(dataset_labs):\n",
    "        a = dataset_labs.iloc[row,col1]\n",
    "        b = dataset_labs.iloc[row,col2]\n",
    "        if a==b or (np.isnan(a) and np.isnan(b)):\n",
    "            pass\n",
    "        elif a!=b:\n",
    "            changed +=1\n",
    "            avg = (a+b)/2\n",
    "            dataset_labs.iloc[row,col1] = avg\n",
    "            if (np.isnan(a) and ~np.isnan(b)) or (np.isnan(b) and ~np.isnan(a)):\n",
    "                null_l.append(row)\n",
    "        else:\n",
    "            print(a)\n",
    "            print(b)\n",
    "        row +=1       \n",
    "    # delete the redundant column max, update counters\n",
    "    dataset_labs.drop(dataset_labs.columns[col2], axis=1, inplace = True)\n",
    "    counter = counter+1\n",
    "    col1 = col1+1\n",
    "    col2 = col2+1\n",
    "\n",
    "dataset_labs.columns = ['subject_id','hadm_id', 'icustay_id', 'charttime', 'aniongap_avg', 'bicarbonate_avg', \n",
    "                        'creatinine_avg', 'chloride_avg', 'glucose_avg', 'hematocrit_avg','hemoglobin_avg',\n",
    "                        'potassium_avg', 'sodium_avg', 'bun_avg', 'wbc_avg']\n",
    "if len(null_l)>0:\n",
    "    print(\"null values encountered\")\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Merge creatinine and glucose.\")\n",
    "# merge creatinine from labs and set with labels\n",
    "creat_l = dataset_labs[['icustay_id','charttime','creatinine_avg']].copy()\n",
    "creat_l = creat_l.dropna(subset=['creatinine_avg'])\n",
    "creat = X[['icustay_id','charttime', 'creat']].copy()\n",
    "creat = creat.dropna(subset=['creat'])\n",
    "creat_l = creat_l.rename(columns={\"creatinine_avg\": \"creat\"})\n",
    "creat = creat.append(creat_l, ignore_index=True)\n",
    "creat.drop_duplicates(inplace = True)\n",
    "#delete old columns\n",
    "dataset_labs.drop([\"creatinine_avg\"], axis = 1, inplace = True)\n",
    "dataset_labs = dataset_labs.dropna(subset=dataset_labs.columns[4:], how='all')\n",
    "X.drop([\"creat\"], axis = 1, inplace = True)\n",
    "#merge new column\n",
    "X = pd.merge(X, creat, on = [\"icustay_id\", \"charttime\"], sort = True, how= \"outer\", copy = False)\n",
    "\n",
    "if MAX_FEATURE_SET:\n",
    "    # merge glucose from vitals and labs\n",
    "    glucose_v = dataset_vitals[['subject_id','hadm_id','icustay_id','charttime', 'glucose_mean']].copy()\n",
    "    glucose_v = glucose_v.dropna(subset=['glucose_mean'])\n",
    "    glucose = dataset_labs[['subject_id','hadm_id','icustay_id','charttime', 'glucose_avg']].copy()\n",
    "    glucose = glucose.dropna(subset=['glucose_avg'])\n",
    "    glucose_v = glucose_v.rename(columns={\"glucose_mean\": \"glucose_avg\"})\n",
    "    glucose = glucose.append(glucose_v, ignore_index=True)\n",
    "    glucose.drop_duplicates(inplace = True)\n",
    "    #delete old columns\n",
    "    dataset_labs.drop([\"glucose_avg\"], axis = 1, inplace = True)\n",
    "    dataset_vitals.drop([\"glucose_mean\"], axis = 1, inplace = True)\n",
    "    dataset_vitals = dataset_vitals.dropna(subset=dataset_vitals.columns[4:], how='all')\n",
    "    #merge new column\n",
    "    dataset_labs = pd.merge(dataset_labs, glucose, on = ['subject_id','hadm_id','icustay_id','charttime',], sort = True, how= \"outer\", copy = False)\n",
    "    \n",
    "dataset_labs = dataset_labs.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "X = X.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Merging labs, vitals and vents files\")\n",
    "if MAX_FEATURE_SET:\n",
    "    X = pd.merge(X, dataset_labs, on = [\"icustay_id\", \"charttime\"], how= \"outer\", copy = False)\n",
    "    X = pd.merge(X, dataset_vitals, on = [\"icustay_id\", \"charttime\",\"subject_id\", \"hadm_id\"], how= \"outer\", copy = False)\n",
    "    X = pd.merge(X, dataset_vents, on = [\"icustay_id\", \"charttime\"], how= \"outer\", copy = False) \n",
    "    X.drop([\"subject_id\"], axis = 1, inplace = True) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"start preprocessing time dependent data\") \n",
    "print(\"Removing patients under the min age\")\n",
    "dataset_detail = dataset_detail.loc[dataset_detail['age'] >= ADULTS_MIN_AGE]\n",
    "adults_icustay_id_list = dataset_detail['icustay_id'].unique()\n",
    "X = X[X.icustay_id.isin(adults_icustay_id_list)].sort_values(by=['icustay_id'], ignore_index = True)\n",
    "X = X.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "adults_icustay_id_list = np.sort(adults_icustay_id_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"drop icustay_id with time span less than 48hrs\")\n",
    "def more_than_HOURS_ahead(adults_icustay_id_list, X):\n",
    "    drop_list = []\n",
    "    los_list = [] # calculating LOS ICU based on charttime\n",
    "    long_stays_id = [] # LOS longer than MAX DAYS days\n",
    "    last_charttime_list = []\n",
    "    seq_length = X.groupby(['icustay_id'],as_index=False).size()\n",
    "    id_count = 0\n",
    "    first_row_index = 0\n",
    "\n",
    "    while id_count < len(adults_icustay_id_list):\n",
    "        icustay_id = adults_icustay_id_list[id_count]\n",
    "        last_row_index = first_row_index + seq_length.iloc[id_count,1]-1\n",
    "        first_time = X.iat[first_row_index, X.columns.get_loc('charttime')]\n",
    "        last_time = X.iat[last_row_index, X.columns.get_loc('charttime')]\n",
    "        los = round(float((last_time - first_time).total_seconds()/60/60/24),4) # in days\n",
    "        if los < HOURS_AHEAD/24:\n",
    "            drop_list.append(icustay_id)\n",
    "        else:\n",
    "            los_list.append(los)\n",
    "            if los > MAX_DAYS:\n",
    "                long_stays_id.append(icustay_id)\n",
    "                last_charttime_list.append(last_time)\n",
    "        # udpate for the next icustay_id\n",
    "        first_row_index = last_row_index+1\n",
    "        id_count +=1\n",
    "    if len(long_stays_id) != len(last_charttime_list):\n",
    "        print('ERROR')\n",
    "    print(\"%d long stays\" % len(long_stays_id))\n",
    "    # drop all the rows with the saved icustay_id\n",
    "    print(\"there are %d id-s shorter than 48 hours\" % len(drop_list))\n",
    "    X = X[~X.icustay_id.isin(drop_list)]\n",
    "    id_list = X['icustay_id'].unique()\n",
    "    X = X.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "    \n",
    "    return id_list, X, long_stays_id,last_charttime_list\n",
    "\n",
    "id_list, X, long_stays_id,last_charttime_list  = more_than_HOURS_ahead(adults_icustay_id_list, X)\n",
    "\n",
    "long = pd.DataFrame()\n",
    "long['icustay_id']  = long_stays_id\n",
    "long['last_time']  = last_charttime_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deleting rows that are not within MAX_DAYS (35) period\n",
    "i = 0 # long df index\n",
    "drop_long_time = []\n",
    "    \n",
    "while i < len(long_stays_id):\n",
    "    j = 0\n",
    "    all_rows = X.index[X['icustay_id'] == long.loc[i,'icustay_id']].tolist()\n",
    "    while j < len(all_rows):\n",
    "        time = X.iat[all_rows[j], X.columns.get_loc('charttime')]\n",
    "        # if keep last MAX_DAYS \n",
    "        if (long.loc[i,'last_time'] - time).total_seconds() > MAX_DAYS*24*60*60:\n",
    "            drop_long_time.append(all_rows[j])\n",
    "            j +=1\n",
    "        else:\n",
    "            break\n",
    "    i +=1       \n",
    "X.drop(X.index[drop_long_time], inplace=True) \n",
    "\n",
    "# checking for 48h min length again\n",
    "id_list, X, long_stays_id,last_charttime_list  = more_than_HOURS_ahead(id_list, X)\n",
    "dataset_detail = dataset_detail[dataset_detail.icustay_id.isin(id_list)].sort_values(by=['icustay_id'], ignore_index = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For testing purpose, use small amount of data first\n",
    "if TESTING:\n",
    "    rest, id_list = train_test_split(id_list, test_size= TEST_SIZE, random_state=42)\n",
    "    X = X[X.icustay_id.isin(id_list)].sort_values(by=['icustay_id'])\n",
    "    dataset_detail = dataset_detail[dataset_detail.icustay_id.isin(id_list)].sort_values(by=['icustay_id'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resampling , imputing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if (TIME_SAMPLING and MOST_COMMON):\n",
    "    print(\"resampling: MOST_COMMON\")\n",
    "    # Resample the data using assigned interval,mode() for most common\n",
    "    X = X.set_index('charttime').groupby('icustay_id').resample(SAMPLING_INTERVAL).mode().reset_index()\n",
    "elif TIME_SAMPLING:\n",
    "    print(\"resampling: MEAN & ZERO\")\n",
    "    # Sampling with different strategies per kind of variable\n",
    "    label = ['aki_stage']\n",
    "    skip = ['icustay_id', 'charttime', 'aki_stage']\n",
    "    if MAX_FEATURE_SET:\n",
    "        discrete_feat = ['sedative', 'vasopressor', 'vent', 'hadm_id']\n",
    "        skip.extend(discrete_feat)    \n",
    "    # all features that are not in skip are numeric\n",
    "    numeric_feat = list(X.columns.difference(skip))\n",
    "    \n",
    "    # Applying aggregation to features depending on their type\n",
    "    X = X.set_index('charttime').groupby('icustay_id').resample(SAMPLING_INTERVAL)\n",
    "    if MAX_FEATURE_SET:\n",
    "        X_discrete = X[discrete_feat].max().fillna(FILL_VALUE).astype(np.int64)\n",
    "    X_numeric = X[numeric_feat].mean() \n",
    "    X_label = X['aki_stage'].max()\n",
    "    print(\"Merging sampled features\")\n",
    "    try:\n",
    "        X = pd.concat([X_numeric, X_discrete,X_label], axis=1).reset_index()\n",
    "    except:\n",
    "        X = pd.concat([X_numeric,X_label], axis=1).reset_index()\n",
    "print(X.shape)\n",
    "#Label forward fill\n",
    "X['aki_stage'] = X['aki_stage'].ffill(limit=RESAMPLE_LIMIT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Imputation.\")\n",
    "# do imputation of label with zero if there are still missing values\n",
    "X['aki_stage'] = X['aki_stage'].fillna(0)\n",
    "\n",
    "# using most common within each icustay_id\n",
    "if IMPUTE_EACH_ID:\n",
    "    column_name = list(X.columns)\n",
    "    column_name.remove(column_name[0]) \n",
    "    for feature in column_name:\n",
    "        X.loc[X[feature].isnull(), feature] = X.icustay_id.map(fast_mode(X, ['icustay_id'], feature).set_index('icustay_id')[feature])       \n",
    "\n",
    "# imputation based on whole column\n",
    "if IMPUTE_COLUMN:\n",
    "    imp = SimpleImputer(missing_values=np.nan, strategy= IMPUTE_METHOD)\n",
    "    cols = list(X.columns)\n",
    "    cols = cols[2:23]\n",
    "    X[cols]=imp.fit_transform(X[cols])  \n",
    "\n",
    "# If no imputation method selected or only impute each id, for the remaining nan impute direclty with FILL_VALUE\n",
    "X = X.fillna(FILL_VALUE) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# more comfortable to review in this order\n",
    "try:\n",
    "    cols = ['icustay_id', 'charttime','aki_stage','hadm_id','aniongap_avg','bicarbonate_avg', 'bun_avg','chloride_avg',\n",
    "            'creat','diasbp_mean', 'glucose_avg', 'heartrate_mean', 'hematocrit_avg','hemoglobin_avg', \n",
    "            'potassium_avg', 'resprate_mean', 'sodium_avg','spo2_mean', 'sysbp_mean', 'uo_rt_12hr', \n",
    "            'uo_rt_24hr', 'uo_rt_6hr','wbc_avg', 'sedative', 'vasopressor', 'vent' ]\n",
    "    X = X[cols]\n",
    "    print(\"success\")\n",
    "except:\n",
    "    try:\n",
    "        cols = ['icustay_id', 'charttime','aki_stage','creat','uo_rt_12hr', 'uo_rt_24hr', 'uo_rt_6hr']\n",
    "        X = X[cols]\n",
    "    except:\n",
    "        print(\"error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"binarise labels\")\n",
    "if ALL_STAGES:\n",
    "    pass\n",
    "elif CLASS1:\n",
    "    X.loc[X['aki_stage'] > 1, 'aki_stage'] = 1\n",
    "elif CLASS2:\n",
    "    X.loc[X['aki_stage'] < 2, 'aki_stage'] = 0\n",
    "    X.loc[X['aki_stage'] > 1, 'aki_stage'] = 1\n",
    "elif CLASS3:\n",
    "    X.loc[X['aki_stage'] < 3, 'aki_stage'] = 0\n",
    "    X.loc[X['aki_stage'] > 2, 'aki_stage'] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SHIFTING labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"Shifting the labels 48 h\") # by 8 position : 6h sampling*8=48h and ffil 8 newly empty ones\n",
    "X['aki_stage'] = X.groupby('icustay_id')['aki_stage'].shift(-(HOURS_AHEAD // int(SAMPLING_INTERVAL[:-1])))\n",
    "X = X.dropna(subset=['aki_stage'])\n",
    "X['icustay_id'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add categorical features (details)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"start preprocessing not time dependent data\")\n",
    "if MAX_FEATURE_SET:\n",
    "    #extract datasets based on id_list\n",
    "    dataset_detail = dataset_detail.loc[dataset_detail['icustay_id'].isin(id_list)]\n",
    "    #sort by ascending order\n",
    "    dataset_detail = dataset_detail.sort_values(by=['icustay_id'])\n",
    "    subject_id = dataset_detail[\"subject_id\"].unique()\n",
    "    #transfrom categorical data to binary form\n",
    "    dataset_detail = dataset_detail.join(pd.get_dummies(dataset_detail.pop('gender')))\n",
    "    dataset_detail = dataset_detail.join(pd.get_dummies(dataset_detail.pop(\"ethnicity_grouped\")))\n",
    "    dataset_detail = dataset_detail.join(pd.get_dummies(dataset_detail.pop('admission_type')))\n",
    "    dataset_detail = dataset_detail.drop(['subject_id', 'hadm_id'], axis=1)\n",
    "    X =  pd.merge(X, dataset_detail, on = [\"icustay_id\"], how= \"left\", copy = False) \n",
    "    numeric_feat.append('age')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names =['Anion gap', 'Bicarbonate', 'Blood Urea Nitrogen', 'Chloride', 'Creatinine', 'Diastolic BP', 'Glucose', 'Heart rate', \n",
    "            'Hematocrit', 'Hemoglobin', 'Potassium', 'Respiratory rate', 'Sodium', 'Oxygen saturation', 'Systolic BP', 'Urine output 12h', 'Urine output 24h', 'Urine output 6h',\n",
    "            'White cell count', 'Sedative', 'Vasopressor', 'Ventilation', 'Age', 'Female gender', 'Male gender', 'Asian ethnicity', 'Black ethnicity', 'Hispanic ethnicity', 'Native american', \n",
    "            'Other ethnicity', 'Ethnicity unknown', 'White ethnicity', 'Elective admission', 'Emergency admission', 'Urgent admission']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Cap features between 0.01 / 0.99 quantile and normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = cap_data(X)\n",
    "\n",
    "X = normalise_data(X, numeric_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X.sort_values(by=['icustay_id', 'charttime'])\n",
    "seq_lengths = X.groupby(['icustay_id'],as_index=False).size().sort_values(by = ['size'],ascending=False)\n",
    "sequence_length = seq_lengths.max() # the longest sequence per icustay-id\n",
    "print(sequence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#AL re-write as try except to make it work as hadm_id is not used if only one csv file is used and none are merged\n",
    "try:\n",
    "    X.drop(['hadm_id'], axis=1, inplace = True)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "features = X.shape[1]-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"divide dataset into train, test and validation sets\")\n",
    "id_train, id_test_val = train_test_split(id_list, test_size = SPLIT_SIZE, random_state = 42) # train set is 80%)\n",
    "print(\"train is %d\" % len(id_train))\n",
    "# remaining 20% split in halves as test and validation 10% and 10%\n",
    "id_valid, id_test = train_test_split(id_test_val, test_size = 0.5, random_state = 42) # test 10% valid 10%\n",
    "print(\"val and test are %d\" %len(id_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = X[X.icustay_id.isin(id_train)].sort_values(by=['icustay_id'])\n",
    "test = X[X.icustay_id.isin(id_test)].sort_values(by=['icustay_id'], ignore_index = True) \n",
    "validation = X[X.icustay_id.isin(id_valid)].sort_values(by=['icustay_id']) \n",
    "\n",
    "test = test.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "train = train.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "validation = validation.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['icustay_id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_seq = test.groupby(['icustay_id'],as_index=False).size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate the test set before onset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z = test.copy(deep = True)\n",
    "test = test.sort_values(by=['icustay_id', 'charttime'], ignore_index = True)\n",
    "id_test.sort()\n",
    "#last_charttime_list= []\n",
    "\n",
    "index_list = []\n",
    "label_list = []\n",
    "\n",
    "first_row_index = 0\n",
    "id_count = 0\n",
    "seq_length = Z.groupby(['icustay_id'],as_index=False).size()\n",
    "\n",
    "for ID in id_test:\n",
    "    last_row_index = first_row_index + seq_length.iloc[id_count,1]-1\n",
    "    a = Z.loc[Z['icustay_id']==ID].aki_stage\n",
    "    if 1 not in a.values:\n",
    "        label_list.append(0)\n",
    "        #last_charttime_list.append(Z.iat[last_row_index, Z.columns.get_loc('charttime')]) \n",
    "        index_list.append(last_row_index)\n",
    "    elif 1 in a.values:\n",
    "        label_list.append(1)\n",
    "        row = first_row_index\n",
    "        while row != last_row_index+1:\n",
    "            if Z.iat[row, Z.columns.get_loc('aki_stage')]==0:\n",
    "                row +=1\n",
    "            elif Z.iat[row, Z.columns.get_loc('aki_stage')]==1:\n",
    "                #last_charttime_list.append(Z.iat[row, Z.columns.get_loc('charttime')])\n",
    "                index_list.append(row)\n",
    "                break\n",
    "    first_row_index = last_row_index+1\n",
    "    id_count +=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.drop(['charttime'], axis=1, inplace = True)\n",
    "train.drop(['charttime'], axis=1, inplace = True)\n",
    "validation.drop(['charttime'], axis=1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.version.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"reshape 2D dataframe to 3D Array, group by icustay_id\")\n",
    "train = np.array(sorted(list(train.groupby(['icustay_id'],as_index=False).apply(pd.DataFrame.to_numpy)),key=len, reverse=True))\n",
    "test = np.array(list(test.groupby(['icustay_id'],as_index=False).apply(pd.DataFrame.to_numpy)))\n",
    "validation = np.array(list(validation.groupby(['icustay_id'],as_index=False).apply(pd.DataFrame.to_numpy)))\n",
    "\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(validation.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (torch.cuda.is_available()):\n",
    "    print('Training on GPU')\n",
    "else:\n",
    "    print('Training on CPU') # On mac book GPU is not possible =() \n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def batch(data, batch_size):\n",
    "    X_batches = []\n",
    "    y_batches = []\n",
    "    times = math.floor(data.shape[0]/batch_size)\n",
    "    remainder = data.shape[0]%times\n",
    "    a = 0\n",
    "    start = 0\n",
    "    end = start+batch_size\n",
    "    if remainder ==0:\n",
    "        a +=1\n",
    "    while a<times:\n",
    "        temp = pad(data[start:end,],0)\n",
    "        x = torch.from_numpy(temp[:,:,2:]).float() # without icustay_id and without aki_stage columns\n",
    "        y = torch.flatten(torch.from_numpy(temp[:, :,1].reshape(-1,1)).float()).long()\n",
    "        X_batches.append(x)\n",
    "        y_batches.append(y)\n",
    "        start = end\n",
    "        end = start+batch_size\n",
    "        a +=1\n",
    "    temp = pad(data[start:data.shape[0]],0)\n",
    "    x = torch.from_numpy(temp[:,:,2:]).float()\n",
    "    y = torch.flatten(torch.from_numpy(temp[:, :,1].reshape(-1,1)).float()).long()\n",
    "    X_batches.append(x)\n",
    "    y_batches.append(y)\n",
    "    if len(X_batches) != len(y_batches):\n",
    "        print(\"length error\")\n",
    "    return X_batches, y_batches # arrays\n",
    "\n",
    "# batching\n",
    "X_train, y_train = batch(train, batch_size) # to count weights\n",
    "\n",
    "# counting balance of the classes\n",
    "y = []\n",
    "for i in y_train:\n",
    "    for element in i:\n",
    "        y.append(element.item())\n",
    "\n",
    "#  weights\n",
    "counter=collections.Counter(y)\n",
    "print(counter)\n",
    "zeroes = counter[0]\n",
    "ones = counter[1]\n",
    "\n",
    "X_test, y_test = batch(test, test.shape[0]) \n",
    "X_val, y_val = batch(validation, validation.shape[0])\n",
    "X_val = X_val[0]\n",
    "y_val = y_val[0]\n",
    "X_test = X_test[0]\n",
    "y_test = y_test[0]\n",
    "print(y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################\n",
    "# setup\n",
    "\n",
    "bi_directional = True\n",
    "n_epochs = 150\n",
    "lr = 0.001\n",
    "emb_size = round(features/1)\n",
    "number_layers = 3\n",
    "dropout = 0 # dropout\n",
    "\n",
    "##########################\n",
    "input_size = features\n",
    "output_size = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size, emb_size, output_size, bi_directional, number_layers, dropout):\n",
    "        super(Net, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.emb_size = emb_size \n",
    "        self.output_size = output_size\n",
    "        self.number_layers = number_layers\n",
    "        self.fc1 = nn.Linear(self.input_size, self.emb_size, bias = True) # I can have a few (IV) within this line - documentation        \n",
    "        self.fc2 = nn.LSTM(self.emb_size, self.output_size,num_layers=self.number_layers, batch_first = True, bidirectional = bi_directional) \n",
    "        # in bidirectional encoder we have  forward and backward hidden states\n",
    "        self.encoding_size = self.output_size * 2 if bi_directional else self.output_size\n",
    "        self.combination_layer = nn.Linear(self.encoding_size, self.encoding_size)\n",
    "        # Create affine layer to project to the classes \n",
    "        self.projection = nn.Linear(self.encoding_size, self.output_size)\n",
    "        #dropout layer for regularizetion of a sequence\n",
    "        self.dropout_layer = nn.Dropout(p = dropout)  \n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        h = self.relu(self.fc1(x))\n",
    "        h, _ = self.fc2(h) # h, _ : as I have 2outputs (tuple), only take the real output [0]. \n",
    "        #print(type(h)) # Underscore throughs away the rest, _ \"I do not care\" variable notation in python\n",
    "        h = self.relu(self.combination_layer(h))\n",
    "        h = self.dropout_layer(h)\n",
    "        h = self.projection(h) \n",
    "        return h\n",
    "\n",
    "#create a network \n",
    "nn_model = Net(input_size, emb_size, output_size,bi_directional, number_layers, dropout)\n",
    "#print(nn_model)\n",
    "#print(list(nn_model.parameters()))\n",
    "\n",
    "\n",
    "# BCE Loss and optimizer\n",
    "criterion = nn.BCEWithLogitsLoss(pos_weight = torch.tensor(round(zeroes/ones,0))) # class imbalance\n",
    "#print(round(zeroes/ones,0))\n",
    "optimizer = optim.Adam(nn_model.parameters(), lr=lr) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# training loop (full data 3.5 hours)\n",
    "\n",
    "epochs = n_epochs\n",
    "starttime = datetime.now() # datetime object containing current date and time\n",
    "train_losses, validation_losses = [], []\n",
    "best = 0\n",
    "\n",
    "for epoch in range(epochs):  # loop over the dataset multiple times\n",
    "    print (\"\\n Epoch [%d] out of %d\" % (epoch + 1, epochs))\n",
    "    running_loss = 0.0\n",
    "    validation_loss = 0.0\n",
    "    roc_auc = 0.0\n",
    "    pr_auc = 0.0\n",
    "    m = 0\n",
    "    \n",
    "    #train\n",
    "    #print(list(nn_model.parameters())[0])\n",
    "    for i in X_train:\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad() # zero the gradient buffers not to consider gradients of previous iterations\n",
    "        X_batch = X_train[m]\n",
    "        y_batch = y_train[m]\n",
    "        # forward + backward + optimize\n",
    "        outputs = nn_model(X_batch)\n",
    "        outputs = torch.flatten(outputs)\n",
    "        y_batch = y_batch.type_as(outputs)\n",
    "        loss = criterion(outputs, y_batch)\n",
    "        loss.backward()\n",
    "        optimizer.step() # Does the update\n",
    "        running_loss += loss.item()\n",
    "        m +=1\n",
    "        \n",
    "   \n",
    "        #validation \n",
    "    nn_model.eval()\n",
    "    with torch.no_grad():\n",
    "        v_out = nn_model(X_val) \n",
    "        v_out = torch.flatten(v_out) \n",
    "        y_val = y_val.type_as(v_out)\n",
    "        v_loss = criterion(v_out, y_val)\n",
    "        validation_loss = v_loss.item()\n",
    "        # auc and pr auc\n",
    "        val_prob = torch.nn.Sigmoid() (v_out)\n",
    "        roc_auc = roc_auc_score(y_val,val_prob) \n",
    "        \n",
    "    validation_losses.append(validation_loss) \n",
    "    train_losses.append(running_loss/len(X_train)) \n",
    "    print(f\"Training loss: {running_loss/len(X_train):.3f}.. \" f\"Validation loss: {validation_loss:.3f}.. \")\n",
    "    print(f\"AUC: {roc_auc:.2f}\")  \n",
    "    nn_model.train()\n",
    "    \n",
    "    \n",
    "    if roc_auc > best:\n",
    "        best = roc_auc\n",
    "        PATH = './LSTMbest.pth' \n",
    "        torch.save(nn_model.state_dict(), PATH) # save the model\n",
    "    else:\n",
    "        pass\n",
    "    \n",
    "       \n",
    "print('Finished Training')\n",
    "print(\"starttime =\", starttime)\n",
    "now = datetime.now()\n",
    "print(\"now =\", now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = './i-Bidir_3_lr_0.001_nodropbest.pth'\n",
    "\n",
    "# save the model\n",
    "#torch.save(nn_model.state_dict(), PATH)\n",
    "\n",
    "# code to load saved model\n",
    "nn_model = Net(input_size, emb_size, output_size,bi_directional, number_layers, dropout)\n",
    "nn_model.load_state_dict(torch.load(PATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(y_test) # single batch with zero padding to the max shape 635208"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Next step testing the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Continuous performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = nn_model(X_test)\n",
    "pred = torch.nn.Sigmoid() (logits)\n",
    "pred = pred.detach().numpy()\n",
    "pred = pred.reshape(-1,1)\n",
    "print(\"Performance on full X_test where it has no batching: is padded to max dimentions. \\n\")\n",
    "print (\"Area Under ROC Curve: %0.2f\" % roc_auc_score(y_test, pred, average = 'micro')  )\n",
    "brier = round(metrics.brier_score_loss(y_test, pred, sample_weight=None, pos_label=None),3)\n",
    "print(\"Brier score : {:.3f}\".format(brier))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('padded_lstm.npy', 'wb') as f:\n",
    "    np.save(f, y_test)\n",
    "    np.save(f, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timestamps = X_test.shape[1] #133\n",
    "icustays = X_test.shape[0]\n",
    "times = []\n",
    "auc_s = []\n",
    "t = 0\n",
    "\n",
    "while t < timestamps:\n",
    "    times.append(t+1)\n",
    "    row = t\n",
    "    i = 0\n",
    "    prob_t = []\n",
    "    y_t = []\n",
    "    while i < icustays:\n",
    "        prob_t.append(pred[row])\n",
    "        y_t.append(y_test[row])\n",
    "        row += timestamps\n",
    "        i +=1\n",
    "    prob_t = np.array(prob_t).reshape(-1,1)\n",
    "    y_t = np.array(y_t).reshape(-1,1)\n",
    "    auc_s.append(roc_auc_score(y_t, prob_t, average = 'micro'))\n",
    "    t +=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df =  pd.DataFrame(auc_s, columns = ['AUC'])\n",
    "df['Timestamps'] = times\n",
    "#df[120:133]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot \n",
    "sns.lineplot(x=\"Timestamps\", y=\"AUC\", color = 'g',\n",
    "             data=df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing to LogR, XGB, RF models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, y_test = batch(test, test.shape[0]) \n",
    "X_test = X_test[0]\n",
    "y_test = y_test[0]\n",
    "\n",
    "\n",
    "def to_one_label (model, label_list,X_test,index_list):\n",
    "    # evaluate on a test set\n",
    "    labels = np.array(label_list)\n",
    "    labels = labels.reshape(-1,1)\n",
    "    labels = labels.astype(int)\n",
    "    logits = model(X_test)\n",
    "    pred = torch.nn.Sigmoid() (logits)\n",
    "    max_rows = pred.shape[1]\n",
    "    predictions = pred.detach().numpy()\n",
    "    predictions = predictions.reshape(-1,1) \n",
    "    # select 1 per icu stay id by index\n",
    "    prob_1_label = []\n",
    "    row = 0\n",
    "    prev = 0\n",
    "    for i in index_list:\n",
    "        prob_1_label.append(predictions[row+i-prev])\n",
    "        row += pred.shape[1]\n",
    "        prev = i\n",
    "    prob_1_label = np.array(prob_1_label).reshape(-1,1)\n",
    "    \n",
    "    return labels, prob_1_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def performance (y_test, pred_probabilities):\n",
    "    # performance\n",
    "    fpr, tpr, thresholds = roc_curve(y_test, pred_probabilities)\n",
    "    # compute roc auc\n",
    "    roc_auc = roc_auc_score(y_test, pred_probabilities, average = 'micro')\n",
    "    # compute Precision_Recall curves\n",
    "    precision, recall, _ = precision_recall_curve(y_test, pred_probabilities)\n",
    "    # compute PR_AUC\n",
    "    pr_auc = metrics.auc(recall, precision)\n",
    "       \n",
    "    # I add confusion matrix\n",
    "    optimal_cut_off = round(thresholds[np.argmax(tpr - fpr)],4)\n",
    "    a = np.where(pred_probabilities > optimal_cut_off, 1, 0)\n",
    "    brier = round(metrics.brier_score_loss(y_test, pred_probabilities, sample_weight=None, pos_label=None),3)\n",
    "    predictions = np.where(pred_probabilities > optimal_cut_off, 1, 0)  \n",
    "    \n",
    "    print (\"Area Under ROC Curve: %0.2f\" % roc_auc  )\n",
    "    #print (\"Area Under PR Curve(AP): %0.2f\" % pr_auc  ) \n",
    "    print(\"Brier score : {:.3f}\".format(brier))\n",
    "    #print('Accuracy for Classifier : {:.2f}'.format(accuracy_score(y_test, predictions)))\n",
    "    #print('Cut off: ' + str(optimal_cut_off))\n",
    "    matrix = metrics.confusion_matrix(y_test, a, labels=None, normalize=None)\n",
    "    #print(str(matrix))\n",
    "    \n",
    "    #f.write(\"\\n Area Under ROC Curve: \" +str(roc_auc))\n",
    "    #f.write(\"\\n Area Under PR Curve(AP): \" + str(pr_auc))\n",
    "    #f.write(\"\\n Brier score: \" +str(brier))\n",
    "    #f.write('\\n Accuracy for Classifier '+str(round((accuracy_score(labels, predictions)),3)))\n",
    "    #f.write(\"\\n Cut off: \" +str(optimal_cut_off))\n",
    "    #f.write(str(matrix))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "labels, prob_1_label = to_one_label (nn_model, label_list,X_test,index_list)\n",
    "performance(labels,prob_1_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save labels, prob_1_label\n",
    "\n",
    "with open('test.npy', 'wb') as f:\n",
    "    #np.save(f, labels)\n",
    "    np.save(f, prob_1_label)\n",
    "with open('test.npy', 'rb') as f:\n",
    "    #lstm_labels = np.load(f)\n",
    "    lstm_prob = np.load(f)\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    
    "print(datetime.now())\n",
    "X_test.requires_grad_()\n",
    "attr, delta = ig.attribute(X_test,target=1, return_convergence_delta=True)\n",
    "attr = attr.detach().numpy()\n",
    "attr= np.reshape(attr,(-1,35))\n",
    "importances = np.mean(attr, axis=0)\n",
    "print(datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attr[:,0].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attr[:,4].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_feature_importances(feature_names, importances, title=\"LSTM Average Feature Importances\", axis_title=\"Features\"):\n",
    "    print(title)\n",
    "    i = 0\n",
    "    while i < features:\n",
    "        print(feature_names[i], \": \", '%.3f'%(importances[i]))\n",
    "        i +=1\n",
    "    x_pos = (np.arange(len(feature_names)))\n",
    "    \n",
    "visualize_feature_importances(feature_names, importances)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_df =  pd.DataFrame(importances, columns = ['Feature Importance'])\n",
    "lstm_df['Features'] = feature_names\n",
    "lstm_df = lstm_df.sort_values(by = ['Feature Importance'], ascending = False, ignore_index = True)\n",
    "#lstm_df[\"Feature Importance\"] =  lstm_df[\"Feature Importance\"]\n",
    "#lstm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_df[\"Feature Importance\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ax = sns.barplot(x='Feature Importance', y='Features', data=lstm_df)\n",
    "ax = sns.barplot(x='Feature Importance', y='Features', data=lstm_df, color = 'grey')\n",
    "ax.set_xlabel('Feature Importance', fontsize = 15)\n",
    "ax.set_ylabel(\"Features\",fontsize=15)\n",
    "ax.set_yticklabels(ax.get_ymajorticklabels(), fontsize = 6)\n",
    "plt.title('LSTM feature Importances')\n",
    "plt.savefig('LSTM_feature_importance_grey.png', dpi = 300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_df['abs'] = abs(lstm_df['Feature Importance'])\n",
    "lstm_df = lstm_df.sort_values(by = ['abs'], ascending = False, ignore_index = True)\n",
    "lstm_df_10 = lstm_df.head(10)\n",
    "#lstm_df_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ax = sns.barplot(x='Feature Importance', y='Features', data=lstm_df_10, palette=\"mako\")\n",
    "\n",
    "ax = sns.barplot(x='Feature Importance', y='Features', data=lstm_df_10, color = 'darkgreen')\n",
    "ax.set_xlabel('Feature Importance', fontsize = 15)\n",
    "ax.set_ylabel(\"Features\",fontsize=15)\n",
    "ax.set_yticklabels(ax.get_ymajorticklabels(), fontsize = 10)\n",
    "plt.title('LSTM top 10 features by feature importance')\n",
    "plt.savefig('LSTM_top10_feature_importance_darkgreen.png', dpi = 300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_graphs (y_test,pred_probabilities, classifier_name, plot_name, algorithm):\n",
    "    \n",
    "    def bin_total(y_true, y_prob, n_bins):\n",
    "        bins = np.linspace(0., 1. + 1e-8, n_bins + 1)\n",
    "\n",
    "        # In sklearn.calibration.calibration_curve, the last value in the array is always 0.\n",
    "        binids = np.digitize(y_prob, bins) - 1\n",
    "\n",
    "        return np.bincount(binids, minlength=len(bins))\n",
    "\n",
    "    def missing_bin(bin_array):\n",
    "        midpoint = \" \"    \n",
    "        if bin_array[0]==0:\n",
    "            midpoint = \"5%, \"\n",
    "        if bin_array[1]==0:\n",
    "            midpoint = midpoint + \"15%, \"\n",
    "        if bin_array[2]==0:\n",
    "            midpoint = midpoint + \"25%, \"\n",
    "        if bin_array[3]==0:\n",
    "            midpoint = midpoint + \"35%, \" \n",
    "        if bin_array[4]==0:\n",
    "            midpoint = midpoint + \"45%, \"\n",
    "        if bin_array[5]==0:\n",
    "            midpoint = midpoint + \"55%, \"\n",
    "        if bin_array[6]==0:\n",
    "            midpoint = midpoint + \"65%, \"\n",
    "        if bin_array[7]==0:\n",
    "            midpoint = midpoint + \"75%, \"\n",
    "        if bin_array[8]==0:\n",
    "            midpoint = midpoint + \"85%, \"\n",
    "        if bin_array[9]==0:\n",
    "            midpoint = midpoint + \"95%, \"\n",
    "        return \"The missing bins have midpoint values of \"+ str(midpoint)\n",
    "    \n",
    "    # performance\n",
    "    fpr, tpr, thresholds = roc_curve(y_test, pred_probabilities)\n",
    "    # compute roc auc\n",
    "    roc_auc = roc_auc_score(y_test, pred_probabilities, average = 'micro')\n",
    "    # compute Precision_Recall curves\n",
    "    precision, recall, _ = precision_recall_curve(y_test, pred_probabilities)\n",
    "    # compute PR_AUC\n",
    "    pr_auc = metrics.auc(recall, precision)\n",
    "\n",
    "    # compute calibration curve\n",
    "    LR_y, LR_x = calibration_curve(y_test, pred_probabilities, n_bins=10)\n",
    "    #find out which one are the missing bins\n",
    "    bin_array = bin_total(y_test, pred_probabilities , n_bins=10)\n",
    "    print(missing_bin(bin_array))\n",
    "\n",
    "    print(\"plot curves and save in one png file\")\n",
    "    #save three plots in one png file\n",
    "    fig, (ax1, ax2, ax3) = plt.subplots(3, figsize=(7, 24))\n",
    "    fig.subplots_adjust(wspace=0.3, hspace= 0.3)\n",
    "    fig.suptitle('Evaluation of '+ plot_name)\n",
    "\n",
    "    fpr, tpr, thresholds = roc_curve(y_test, pred_probabilities)\n",
    "    \n",
    "    # plot roc curve\n",
    "    ax1.plot(fpr, tpr,'C2', label=algorithm+\" \"+\"Classifier \" + str(classifier_name) + \", auc=\" +str(round(roc_auc,2)))\n",
    "    ax1.title.set_text('ROC AUC')\n",
    "    ax1.set(xlabel='False Positive Rate', ylabel='True Positive Rate')\n",
    "    ax1.legend(loc=\"lower right\")\n",
    "\n",
    "    # plot PR curve\n",
    "    ax2.plot(recall, precision,'C2', label=algorithm+\" \"+\"Classifier \" + str(classifier_name) + \", auc=\"+str(round(pr_auc,2)))\n",
    "    ax2.title.set_text('PR AUC')\n",
    "    ax2.set(xlabel='Recall', ylabel='Precision')\n",
    "    ax2.legend(loc=\"lower right\")\n",
    "\n",
    "    # plot calibration curve\n",
    "    ax3.plot(LR_x, LR_y, 'C2',marker='o', linewidth=1, label='LR')\n",
    "    line = mlines.Line2D([0, 1], [0, 1], color='black')\n",
    "    transform = ax3.transAxes\n",
    "    line.set_transform(transform)\n",
    "    ax3.add_line(line)\n",
    "    ax3.title.set_text('Calibration plot for '+str(plot_name))\n",
    "    ax3.set(xlabel= 'Predicted probability', ylabel= 'True probability in each bin')\n",
    "    ax3.legend(loc=\"lower right\")\n",
    "\n",
    "    plt.savefig(plot_name+\".png\")\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distribution(pred_probabilities, y_test, dist_name):\n",
    "    #probabilities distributions graphs\n",
    "    true_1 = pd.DataFrame(pred_probabilities, columns=['Predicted probabilities'])\n",
    "    true_1['labels'] = y_test.tolist()\n",
    "    true_0 = true_1.copy(deep = True) \n",
    "    indexNames = true_1[true_1['labels'] == 0].index\n",
    "    true_1.drop(indexNames , inplace=True)\n",
    "    indexNames = true_0[ true_0['labels'] == 1 ].index\n",
    "    true_0.drop(indexNames , inplace=True)\n",
    "    true_1.drop(columns=['labels'], inplace = True)\n",
    "    true_0.drop(columns=['labels'], inplace = True)\n",
    "    \n",
    "    sns.distplot(true_1['Predicted probabilities'], hist = False, kde = True,\n",
    "                 kde_kws = {'shade': True, 'linewidth': 3,\"color\": \"g\"}, label = 'Class 1')\n",
    "    plt.ylabel('Density')\n",
    "    sns.distplot(true_0['Predicted probabilities'], hist = False, kde = True,\n",
    "                     kde_kws = {'shade': True, 'linewidth': 3}, label = 'Class 0')\n",
    "    plt.title('Density Plot'+ dist_name)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "distribution(prob_1_label, labels.flatten(), \" stacked LSTM no imputation \")\n",
    "plt.savefig('stacked_LSTM_bi_NOimp.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_name = \"None vs. Any AKI\"    ###change every time #Moderate vs. Severe #None vs. Any #Others vs. Severe\n",
    "plot_name = \"LSTM NO imputation\"\n",
    "build_graphs(labels.flatten(), prob_1_label.flatten(), classifier_name, plot_name, \"LSTM\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(labels, prob_1_label)\n",
    "fpr, tpr, thresholds = roc_curve(labels, prob_1_label)\n",
    "optimal_cut_off = round(thresholds[np.argmax(tpr - fpr)],2)\n",
    "prediction = np.where(prob_1_label > optimal_cut_off, 1, 0)\n",
    "f1 = f1_score(labels,prediction)\n",
    "prauc =auc(recall, precision)\n",
    "print('F1 = %.3f, PR auc =%.3f' % (f1,prauc))\n",
    "\n",
    "# plot the precision-recall curves\n",
    "no_skill = len(labels[labels==1]) / len(labels)\n",
    "plt.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No Skill')\n",
    "plt.plot(recall,precision, marker='.', label='LSTM')\n",
    "# axis labels\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "# show the legend\n",
    "plt.legend()\n",
    "# show the plot\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameters tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search grid \n",
    "layers = [1,2,3]\n",
    "l_rate = [0.001, 0.0001]\n",
    "drop = [0,0.2]\n",
    "bidirectionality = [True,False]\n",
    "#loops count\n",
    "hypercount = 0\n",
    "# static parameters\n",
    "n_epochs = 80\n",
    "emb_size = round(features/1)\n",
    "input_size = features\n",
    "output_size = 1\n",
    "###############################\n",
    "\n",
    "f = open('lstm_no_imp_uni.txt', 'w+') #change with or without imp\n",
    "\n",
    "for q1 in bidirectionality:\n",
    "    for q2 in layers:\n",
    "        for q3 in drop:\n",
    "            for q4 in l_rate:\n",
    "                hypercount +=1\n",
    "                name = \"i-Bidir_\" if q1 else \"i-Onedir_\"\n",
    "                name = name+str(q2) + \"_lr_\"+str(q4)\n",
    "                name = name+\"_drop\"+str(q3) if q3 == 0.2 else name+\"_nodrop\"\n",
    "                #set parameters\n",
    "                bi_directional = q1\n",
    "                lr = q4\n",
    "                number_layers = q2\n",
    "                dropout = q3 # dropout\n",
    "                print('hypercount: %d' % hypercount)\n",
    "                print('\\n')\n",
    "                print(name)\n",
    "                f.write('\\n\\n' + str(name)+ '\\n\\n')\n",
    "                    \n",
    "                # create the NN\n",
    "                class Net(nn.Module):\n",
    "                    def __init__(self, input_size, emb_size, output_size, bi_directional, number_layers, dropout):\n",
    "                        super(Net, self).__init__()\n",
    "                        self.input_size = input_size\n",
    "                        self.emb_size = emb_size \n",
    "                        self.output_size = output_size\n",
    "                        self.number_layers = number_layers\n",
    "                        self.fc1 = nn.Linear(self.input_size, self.emb_size, bias = True) # I can have a few (IV) within this line - documentation        \n",
    "                        self.fc2 = nn.LSTM(self.emb_size, self.output_size,num_layers=self.number_layers, batch_first = True, bidirectional = bi_directional) \n",
    "                        # in bidirectional encoder we have  forward and backward hidden states\n",
    "                        self.encoding_size = self.output_size * 2 if bi_directional else self.output_size\n",
    "                        self.combination_layer = nn.Linear(self.encoding_size, self.encoding_size)\n",
    "                        # Create affine layer to project to the classes \n",
    "                        self.projection = nn.Linear(self.encoding_size, self.output_size)\n",
    "                        #dropout layer for regularizetion of a sequence\n",
    "                        self.dropout_layer = nn.Dropout(p = dropout)  \n",
    "                        self.relu = nn.ReLU()\n",
    "\n",
    "                    def forward(self, x):\n",
    "                        h = self.relu(self.fc1(x))\n",
    "                        h, _ = self.fc2(h) # h, _ : as I have 2outputs (tuple), only take the real output [0]. \n",
    "                        #print(type(h)) # Underscore throughs away the rest, _ \"I do not care\" variable notation in python\n",
    "                        h = self.relu(self.combination_layer(h))\n",
    "                        h = self.dropout_layer(h)\n",
    "                        h = self.projection(h) \n",
    "                        return h\n",
    "\n",
    "                #create a network \n",
    "                nn_model = Net(input_size, emb_size, output_size,bi_directional, number_layers, dropout)\n",
    "                print(nn_model)\n",
    "                #print(list(nn_model.parameters()))\n",
    "                \n",
    "                # BCE Loss and optimizer\n",
    "                criterion = nn.BCEWithLogitsLoss(pos_weight = torch.tensor(round(zeroes/ones,0))) # class imbalance\n",
    "                #print(round(zeroes/ones,0))\n",
    "                optimizer = optim.Adam(nn_model.parameters(), lr=lr) \n",
    "    \n",
    "    \n",
    "                # TRAINING LOOP \n",
    "                epochs = n_epochs\n",
    "                starttime = datetime.now() # datetime object containing current date and time\n",
    "                train_losses, validation_losses = [], []\n",
    "                best = 0\n",
    "                patience = 0\n",
    "                old_auc = 0\n",
    "                old_pr = 0\n",
    "\n",
    "                for epoch in range(epochs):  # loop over the dataset multiple times\n",
    "                    print (\"\\n Epoch [%d] out of %d\" % (epoch + 1, epochs))\n",
    "                    running_loss = 0.0\n",
    "                    validation_loss = 0.0\n",
    "                    roc_auc = 0.0\n",
    "                    pr_auc = 0.0\n",
    "                    m = 0\n",
    "                    \n",
    "                    #train\n",
    "                    #print(list(nn_model.parameters())[0])\n",
    "                    for i in X_train:\n",
    "                        # zero the parameter gradients\n",
    "                        optimizer.zero_grad() # zero the gradient buffers not to consider gradients of previous iterations\n",
    "                        X_batch = X_train[m]\n",
    "                        y_batch = y_train[m]\n",
    "                        # forward + backward + optimize\n",
    "                        outputs = nn_model(X_batch)\n",
    "                        outputs = torch.flatten(outputs)\n",
    "                        y_batch = y_batch.type_as(outputs)\n",
    "                        loss = criterion(outputs, y_batch)\n",
    "                        loss.backward()\n",
    "                        optimizer.step() # Does the update\n",
    "                        running_loss += loss.item()\n",
    "                        m +=1\n",
    "                    #validation \n",
    "                    nn_model.eval()\n",
    "                    with torch.no_grad():\n",
    "                        v_out = nn_model(X_val) \n",
    "                        v_out = torch.flatten(v_out) \n",
    "                        y_val = y_val.type_as(v_out)\n",
    "                        v_loss = criterion(v_out, y_val)\n",
    "                        validation_loss = v_loss.item()\n",
    "                        # auc and pr auc\n",
    "                        val_prob = torch.nn.Sigmoid() (v_out)\n",
    "                        precision, recall, thresholds = precision_recall_curve(y_val, val_prob)\n",
    "                        pr_auc = auc(recall, precision)\n",
    "                        roc_auc = roc_auc_score(y_val,val_prob) \n",
    "\n",
    "                    validation_losses.append(validation_loss) \n",
    "                    train_losses.append(running_loss/len(X_train)) \n",
    "                    print(f\"Training loss: {running_loss/len(X_train):.3f}.. \" f\"Validation loss: {validation_loss:.3f}.. \")\n",
    "                    print(f\"AUC: {roc_auc:.2f} \" f\"PR AUC: {pr_auc:.2f} \")  \n",
    "                    nn_model.train()\n",
    "\n",
    "                    \n",
    "                    if roc_auc > best:\n",
    "                        best = roc_auc\n",
    "                        PATH1 = './'+str(name)+'best.pth' \n",
    "                        torch.save(nn_model.state_dict(), PATH1) # save the model\n",
    "                    else:\n",
    "                        pass\n",
    "                    \n",
    "                    if roc_auc == old_auc and pr_auc==old_pr:\n",
    "                        patience +=1\n",
    "                    old_auc = roc_auc\n",
    "                    old_pr = pr_auc\n",
    "                    if patience ==10:\n",
    "                        print(\"out of patience\")\n",
    "                        break\n",
    "\n",
    "                print('\\n Finished Training')\n",
    "                print(\"starttime =\", starttime)\n",
    "                now = datetime.now()\n",
    "                print(\"endtime =\", now)\n",
    "                # end of training loop\n",
    "                \n",
    "                PATH2 = './'+str(name)+'last.pth' \n",
    "                torch.save(nn_model.state_dict(), PATH2) # save the model\n",
    "                print('\\n Last model \\n')\n",
    "                labels, probs = to_one_label(nn_model,label_list,X_test,index_list)\n",
    "                performance (nn_model, labels, probs)\n",
    "                \n",
    "                #load the best model\n",
    "                best_model = Net(input_size, emb_size, output_size,bi_directional, number_layers, dropout)\n",
    "                best_model.load_state_dict(torch.load(PATH1))\n",
    "                print('\\n Best model \\n')\n",
    "                labels, probs = to_one_label(best_model,label_list,X_test,index_list)\n",
    "                performance (best_model, labels, probs)\n",
    "f.close() \n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
